from selenium import webdriver
from selenium.webdriver.edge.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.edge.options import Options
from webdriver_manager.microsoft import EdgeChromiumDriverManager
import json
from datetime import datetime

# 初始化 Selenium WebDriver
options = Options()
options.add_argument("--headless")  # 无头模式，不显示浏览器界面
options.add_argument("--disable-gpu")
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")

# 使用 webdriver_manager 自动下载并管理 Edge WebDriver
service = Service(EdgeChromiumDriverManager().install())
driver = webdriver.Edge(service=service, options=options)

def fetch_text_data(url):
    try:
        # 打开网页
        driver.get(url)

        # 等待页面加载完成（可以根据需要调整等待时间）
        driver.implicitly_wait(10)  # 等待10秒

        # 提取网页标题
        title = driver.title

        # 提取正文内容
        text_content = []
        elements = driver.find_elements(By.XPATH, "//h1 | //h2 | //h3 | //h4 | //h5 | //h6 | //p | //a")
        for element in elements:
            text = element.text.strip()
            if text:
                text_content.append(text)

        # 获取当前时间
        current_time = datetime.now().strftime("%d.%m.%Y, %H.%M Uhr")

        # 构建存储格式
        data = {
            "URL": url,
            "类别": "news",
            "标题": title,
            "时间": current_time,
            "正文": "\n".join(text_content),
            "评论": ""
        }

        return data

    except Exception as e:
        print(f"抓取失败: {e}")
        return None

    finally:
        driver.quit()  # 关闭浏览器

def main():
    url = "http://www.oushinet.com"  # 欧洲时报官网的URL
    data = fetch_text_data(url)

    if data:
        # 将数据存储为 JSON 格式
        with open("output.json", "w", encoding="utf-8") as f:
            json.dump(data, f, ensure_ascii=False, indent=4)
        print("数据已成功存储到 output1.json 文件中")
    else:
        print("未能获取到数据")

if __name__ == "__main__":
    main()